"use strict";

const { TaiSpider } = require("tai-spider");
const { JSDOM } = require("jsdom");

class SXGovSpider extends TaiSpider {
  constructor(options = {}) {
    super(options);
    this.name = "sx-gov-page";
    this.debug = true;
    this.page = 1
    this.current = 0
    this.baseUrl = "http://www.sx.gov.cn/col/col1229439406/index.html?uid=6459046&pageNum="
    this.start_urls = [
      "http://www.sx.gov.cn/col/col1229439406/index.html?uid=6459046&pageNum=1",
    ];
  }
  // 目标 获取分页数据
  *parse(response) {
    const contentString = response
      .css(".Con table tbody tr td")
      .get(2)
      .extract()
    var obj = {};
    contentString.match(/groupSize:(\d+).*totalRecord:(\d+).*perPage:(\d+)/).slice(1).forEach((val, idx) => obj[idx == 0 ? 'groupSize' : idx == 1 ? 'totalRecord' : 'perPage'] = parseInt(val));
    console.log(obj);
    this.page = Math.ceil(obj.totalRecord / obj.perPage)
    while (this.current <= this.page) {
      console.log('url---------------------\n', this.baseUrl + this.current)
      // TODO 加3的 原因是 有3个页面不请求数据 一次性拉去的是45条 有个问题存在是 pageNum改变之后，拉取的数据还是前3页的数据 
      // 下一个突破口可以放在 这个问题上 具体看网络请求 
      yield response.follow(this.baseUrl + this.current + 3, this.parsePage)
    }
  }

  *parsePage(response) {
    // 将这段字符串转换成一个 DOM 对象
    let contentString = response.css(".Con-right_list div").get(0).extract();
    const dom = new JSDOM(contentString);
    // 在模拟的浏览器窗口对象中设置全局的 `window` 和 `document` 对象
    const aDoms = dom.window.document.querySelectorAll("a");
    for (let elem of aDoms) {
      console.log('detail url---------------------\n', elem.href)
      yield response.follow(elem.href, this.parseDetail);
    }
  }

  *parseDetail(response) {
    yield {
      title: response.css("table").get(0).css("table tr td").get(0).extract(),
      content: response.css("table").get(1).css("table tr td").get(0).extract(),
      apartment: response
        .css("table")
        .get(2)
        .css("table tr td")
        .get(1)
        .extract(),
      start: response.css("table").get(2).css("table tr td").get(2).extract(),
      end: response.css("table").get(2).css("table tr td").get(3).extract(),
    };
  }
}

module.exports = SXGovSpider;
